import json
from datetime import datetime

import scrapy
from pypinyin import pinyin, Style
# from spidertools.common_pipeline.base_item import convert_dict
from spidertools.utils.time_utils import get_current_date

from commonresources.spider_items.base_item import convert_dict
from commonresources.spider_items.jiangsu.items import JiangSuGongGongZiYuanJiaoYiWangItem


class JiangSuGongGongZiYuanJiaoYiWangSpider(scrapy.Spider):
    """
        江苏公共资源交易网     http://jsggzy.jszwfw.gov.cn/

        全量会修改请求服务器参数，为减少服务器压力，一定要设置大一点的延迟，>1秒
    """
    name = 'JiangSuGongGongZiYuanJiaoYiWang'
    name_zh = "江苏公共资源交易网"
    province = "江苏"
    city = ''
    allowed_domains = ['jsggzy.jszwfw.gov.cn']
    start_urls = ['http://jsggzy.jszwfw.gov.cn/js/xxtype.json', ]

    def __init__(self, full_dose=False, not_full_type=False):
        """        :param full_dose: 是否全量爬取，默认为false
        """
        self.browser_cookie = {}
        self.not_full_type = not_full_type  # """full_type: True:是否提取全量类型 False:提取中标公告和招标公告"""
        self.convert_dict = convert_dict
        self.full_dose = full_dose
        self.detail_type = {}
        self.capacity = 20
        super().__init__()

    def xpath_helper(self, response, xpath):
        result = response.xpath(xpath)
        result_list = [i.xpath('./text()').extract_first() for i in result][1:]
        return result_list

    def parse(self, response):
        if self.full_dose:
            self.capacity = 200
        self.detail_type = response.json()
        yield scrapy.Request(
            url="http://jsggzy.jszwfw.gov.cn/jyxx/tradeInfonew.html",
            callback=self.handle_response_dispatch_request_post,
        )

    def handle_response_dispatch_request_post(self, response):
        list_citys = self.xpath_helper(response, '//select[@id="city"]/option')
        list_types = self.xpath_helper(response, '//div[@id="ywtype"]/a')
        for city in list_citys:
            for type_zhs in list_types:
                type_pinyin = "".join([i[0] for i in (pinyin(type_zhs.strip(), style=Style.FIRST_LETTER))])[0:4]
                if type_pinyin != "nccq":
                    for options in self.detail_type[type_pinyin]:
                        categorynum = options['categorynum']
                        categoryname = options['categoryname']
                        pn = 0
                        if self.not_full_type and "招标" not in categoryname and "结果" not in categoryname and "成交" not in categoryname:
                            continue
                        payloaddata = self.get_payloaddata(city, categorynum, pn, self.capacity)
                        yield scrapy.FormRequest(
                            url="http://jsggzy.jszwfw.gov.cn/inteligentsearch/rest/inteligentSearch/getFullTextData",
                            callback=self.handle_response,
                            dont_filter=True,
                            body=json.dumps(payloaddata),
                            meta={
                                'need_break': False,
                                'project_area': city,
                                'categorynum': categorynum,
                                'construction_type': type_zhs,
                                'info_type': categoryname,
                                'page_pn': pn,
                            },
                        )

    def get_headers(self):
        headers = {
            "Accept": "application/json, text/javascript, */*; q=0.01",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
            "Connection": "keep-alive",
            "Content-Type": "application/json;charset=UTF-8",
            "Host": "jsggzy.jszwfw.gov.cn",
            "Origin": "http://jsggzy.jszwfw.gov.cn",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36 Edg/86.0.622.51",
            "X-Requested-With": "XMLHttpRequest",
        }
        return headers

    def classify(self):
        classify = {
            "jsgc": [
                {"construction_type": '建设工程'},
                {
                    "categorynum": "001001008",
                    "categoryname": "标前预公示"
                },
                {
                    "categorynum": "001001002",
                    "categoryname": "招标公告"
                },
                {
                    "categorynum": "001001003",
                    "categoryname": "未入围公示"
                },
                {
                    "categorynum": "001001004",
                    "categoryname": "评标结果公示"
                },
                {
                    "categorynum": "001001005",
                    "categoryname": "中标结果公示"
                },
                {
                    "categorynum": "001001006",
                    "categoryname": "项目经理变更"
                },
                {
                    "categorynum": "001001009",
                    "categoryname": "评定分离"
                }
            ],
            "zfcg": [
                {"construction_type": '政府采购'},
                {
                    "categorynum": "001002001",
                    "categoryname": "采购预告"
                },
                {
                    "categorynum": "001002002",
                    "categoryname": "采购公告"
                },
                {
                    "categorynum": "001002003",
                    "categoryname": "变更公告"
                },
                {
                    "categorynum": "001002004",
                    "categoryname": "成交公告"
                },
                {
                    "categorynum": "001002005",
                    "categoryname": "采购方式公示"
                },
                {
                    "categorynum": "001002006",
                    "categoryname": "合同公示"
                }
            ],
            "gycq": [
                {"construction_type": '国有产权'},
                {
                    "categorynum": "001004001",
                    "categoryname": "转让公告"
                },
                {
                    "categorynum": "001004002",
                    "categoryname": "司法拍卖公告"
                },
                {
                    "categorynum": "001004003",
                    "categoryname": "成交公告"
                }
            ],
            "tdykeq": [
                {"construction_type": '土地与矿业权'},
                {
                    "categorynum": "001003001",
                    "categoryname": "出让公告"
                },
                {
                    "categorynum": "001003002",
                    "categoryname": "出让土地实时报价"
                },
                {
                    "categorynum": "001003003",
                    "categoryname": "成交结果公示"
                }
            ],
            "zhjy": [
                {"construction_type": '综合交易'},
                {
                    "categorynum": "001005001",
                    "categoryname": "交易公告"
                },
                {
                    "categorynum": "001005002",
                    "categoryname": "成交公告"
                },
                {
                    "categorynum": "001005003",
                    "categoryname": "涉军交易公告"
                },
                {
                    "categorynum": "001005004",
                    "categoryname": "涉军交易公示"
                }
            ],
            "slgc": [
                {"construction_type": '水利工程'},
                {
                    "categorynum": "001006001",
                    "categoryname": "招标公告"
                },
                {
                    "categorynum": "001006003",
                    "categoryname": "中标候选人公示"
                },
                {
                    "categorynum": "001006002",
                    "categoryname": "中标人公告"
                }

            ]
        }
        return classify

    def get_payloaddata(self, city, categorynum, pn, rn):
        return {
            "accuracy": "",
            "cl": 200,
            "cnum": "001",
            "condition": [{
                "equal": f"{city}",
                "fieldName": "fieldvalue",
                "isLike": "true",
                "likeType": 2
            }, {
                "equal": f"{categorynum}",
                "fieldName": "categorynum",
                "isLike": "true",
                "likeType": 2,
            }],
            "edt": "",
            "exc_wd": "",
            "fields": "title",
            "highlights": "title",
            "inc_wd": "",
            "isBusiness": "1",
            "noParticiple": "1",
            "pn": pn,
            "rn": rn,
            "sdt": "",
            "searchRange": "null",
            "sort": '{"infodatepx":"0"}',
            "ssort": "title",
            "statistics": "null",
            "terminal": "",
            "token": "",
            "unionCondition": "null",
            "wd": "null",
            "time": [{
                "endTime": f"{get_current_date()} 23:59:59",
                "fieldName": "infodatepx",
                "startTime": "2000-10-21 00:00:00",
            }]
        }

    def handle_response(self, response):
        json_obj = response.json()
        if not json_obj['result']['totalcount'] or response.request.meta['page_pn'] > 5000:
            pass
        else:
            for row in json_obj['result']['records']:
                item = dict()
                item['release_time'] = row['infodatepx'][0:10]
                if not self.full_dose and item['release_time'] != get_current_date():
                    response.request.meta['need_break'] = True
                elif item['release_time'] < "2015-12-31":
                    response.meta['need_break'] = True
                else:
                    item['announcement_title'] = row['title']
                    item['origin_url'] = "http://jsggzy.jszwfw.gov.cn" + row['linkurl']
                    item['construction_type'] = response.request.meta['construction_type']
                    item['project_area'] = response.request.meta['project_area']
                    item['info_type'] = response.request.meta['info_type']
                    yield scrapy.Request(url=item['origin_url'],
                                         callback=self.parse_html,
                                         meta=item)

            if not response.request.meta['need_break']:
                url = "http://jsggzy.jszwfw.gov.cn/inteligentsearch/rest/inteligentSearch/getFullTextData"
                page_pn = response.request.meta['page_pn'] + self.capacity
                project_area = response.request.meta['project_area']
                categorynum = response.request.meta['categorynum']
                payloaddata = self.get_payloaddata(project_area, categorynum, page_pn, self.capacity)
                yield scrapy.FormRequest(url=url,
                                         headers=self.get_headers(),
                                         callback=self.handle_response,
                                         body=json.dumps(payloaddata),
                                         meta={
                                             'need_break': False,
                                             'categorynum': categorynum,
                                             'construction_type': response.request.meta['construction_type'],
                                             'info_type': response.request.meta['info_type'],
                                             'project_area': project_area,
                                             'page_pn': page_pn
                                         },
                                         dont_filter=True)

    def parse_html(self, response):
        item = JiangSuGongGongZiYuanJiaoYiWangItem()
        item['html'] = response.text
        item["announcement_title"] = response.meta['announcement_title']
        item["release_time"] = response.meta["release_time"]
        # item['project_area'] = response.meta['project_area']
        item['origin_url'] = response.meta['origin_url']
        """ # 下面两个变量：特殊处理，不是错误（没有弄反），不要在意"""
        item["construction_type"] = response.meta['info_type']
        item['info_type'] = response.meta['construction_type']
        item['is_parsed'] = 0
        if item["construction_type"] in ["成交公告", "结果公示", '成交公示', "采购预告", "采购公告"]:  # 采购类成交公告暂不解析，设置为2
            item['is_parsed'] = 2
        item['source_type'] = self.name_zh
        item['province'] = self.province
        item['project_city'] = response.meta['project_area']
        yield item
